import json
import os
import re
import unicodedata

from bs4 import BeautifulSoup
from urllib3.util.url import parse_url
from utils import ask_url
from spider import Spider


class BookSpider(Spider):
    '''小说爬虫的基类，用于定义小说爬虫的基本属性和方法'''

    def _get_title(self, html):
        '''获取小说标题'''
        soup = BeautifulSoup(html, 'html.parser')
        title = soup.find('h2').text
        return title

    def _get_author(self, html):
        '''获取小说作者'''
        soup = BeautifulSoup(html, 'html.parser')
        author = soup.find('div', class_='info').find(
            'div', class_='small').find_all('span')[0].text
        author = author[3:]
        return author

    def _get_article_id(self, url):
        '''获取小说id'''
        if url.find('/') == -1:
            tid = url.split('.')[0]
        else:
            tid = parse_url(url).path.split('/')[-1].split('.')[0]
        if re.match(r'\d+', tid):
            return tid
        return None

    def _get_contents(self, path):
        for title, url in self._links.items():
            filename = f'{path}/{title}.txt'
            if os.path.exists(filename):
                print(f'已存在文件：{filename}')
                continue
            tid = self._get_article_id(url)
            if tid != None:
                title = title.replace(f'{tid}_', '')
            print(f'正在爬取：{title}, 地址：{url}')
            html = self._get_html(url, self._url)
            item = self._get_content(title, html)
            self.items.append(item)
            self._save_content(filename, item)

    def _get_content(self, title, html):
        '''获取小说章节内容'''
        soup = BeautifulSoup(html, 'html.parser')
        content = soup.find('div', id='content').text
        content = self._remove_content_invalid_chars(content)
        content = unicodedata.normalize('NFKC', content).replace('\r', '\n')
        item = {'title': title, 'content': content}
        return item

    def _remove_content_invalid_chars(self, content):
        '''移除内容中的无效字符'''
        content = content.replace('\xa0', ' ')
        content = content.replace('    ', '')
        content = content.replace('  ', '')
        content = content.replace(' ', '')
        return content

    def _save_content(self, filename, item):
        '''保存小说章节内容到文件'''
        title = item["title"]
        content = item["content"]
        text = f'\n{title}\n\n{content}\n'
        self._save_to_file(filename, text)
        print(f'章节《{title}》保存到文件：{filename}')
        path = os.path.dirname(filename)
        filename = f'{path}/{self.title}.txt'
        self._save_to_file(filename, text, 'a')
        print(f'章节《{title}》追加到文件：{filename}')
        print('----------------------------------------')
